/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package net.crawwle.career.crawler;

import java.net.URL;
import java.util.Calendar;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import net.crawwle.career.crawler.controller.CareerController;
import net.crawwle.career.crawler.util.CrawwleConstants;
import net.htmlparser.jericho.Element;
import net.htmlparser.jericho.HTMLElementName;
import net.htmlparser.jericho.Source;

import com.couchbase.client.CouchbaseClient;
import com.google.gson.Gson;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.url.WebURL;

/**
 * @author Yasser Ganjisaffar <lastname at gmail dot com>
 */
public class CareerCrawler extends WebCrawler {
	/**
	 * Obtains the current CouchbaseClient connection.
	 */

	private final static Pattern FILTERS = Pattern
			.compile(".*(\\.(css|js|bmp|gif|jpe?g"
					+ "|png|tiff?|mid|mp2|mp3|mp4"
					+ "|wav|avi|mov|mpeg|ram|m4v|pdf"
					+ "|rm|smil|wmv|swf|wma|zip|rar|gz))$");

	/**
	 * You should implement this function to specify whether the given url
	 * should be crawled or not (based on your crawling logic).
	 */
	@Override
	public boolean shouldVisit(WebURL url) {
		String href = url.getURL().toLowerCase();
		return !FILTERS.matcher(href).matches();
		// && href.startsWith("http://www.ics.uci.edu/");
	}



	/**
	 * @return couchbaseClient
	 */
	public CouchbaseClient getCouchbaseClient() {
		return ((CareerController)this.getMyController()).getCouchbaseClient();
	}

	/**
	 * @return urlMap
	 */
	public Map<String,String> getUrlMap() {
		return ((CareerController)this.getMyController()).getUrlMap();
	}

	/**
	 * This function is called when a page is fetched and ready to be processed
	 * by your program.
	 */
	@Override
	public void visit(Page page) {
		// CareerSiteInfo doc = new CareerSiteInfo();
		logger.info("fetch page:" + page.getWebURL());

		//int docid = page.getWebURL().getDocid();
		String weburl = page.getWebURL().getURL();
		String domain = page.getWebURL().getDomain();
		// String path = page.getWebURL().getPath();
		String subDomain = page.getWebURL().getSubDomain();
		// String parentUrl = page.getWebURL().getParentUrl();
		//String companyName = page.getWebURL().getAnchor();

		/*
		 * ExtractText extract = new ExtractText(); try {
		 * extract.extractText(weburl); } catch (Exception e) { // TODO 自動生成された
		 * catch ブロック e.printStackTrace(); }
		 */

		//System.out.println("url: " + weburl);
		// System.out.println("URL: " + weburl);
		// System.out.println("Domain: '" + domain + "'");
		// System.out.println("Sub-domain: '" + subDomain + "'");
		// System.out.println("Path: '" + path + "'");
		// System.out.println("Parent page: " + parentUrl);
		// System.out.println("Anchor text: " + companyName);

		try {


			makeCareerInfoByTag(HTMLElementName.LI,weburl, subDomain + "." + domain);
			makeCareerInfoByTag(HTMLElementName.DIV,weburl, subDomain + "." + domain);
			//makeCareerInfoByLiTag(weburl, subDomain + "." + domain);
/*
			if (companyName == null || companyName.indexOf("株式会社") < 0) {
				//makeCareerInfoByLiTag(weburl, subDomain + "." + domain);
				makeCareerInfoByDivTag(weburl, subDomain + "." + domain);
				makeCareerInfoByLiTag(weburl, subDomain + "." + domain);

			} else {
				makeCareerInfoByTableTag(weburl, companyName);
			}*/

		} catch (Exception e) {
			// TODO 自動生成された catch ブロック
			e.printStackTrace();
			logger.warn("fetch error(page=" + weburl + ")\n" + e.getMessage());
		}

		/*
		 * if (page.getParseData() instanceof HtmlParseData) { HtmlParseData
		 * htmlParseData = (HtmlParseData) page.getParseData(); String text =
		 * htmlParseData.getText(); String html = htmlParseData.getHtml();
		 * List<WebURL> links = htmlParseData.getOutgoingUrls();
		 *
		 * System.out.println("Text : " + text);
		 * System.out.println("Html length: " + html.length());
		 * System.out.println("Number of outgoing links: " + links.size());
		 * doc.setText(text); doc.setHtml(html); }
		 *
		 * Header[] responseHeaders = page.getFetchResponseHeaders(); if
		 * (responseHeaders != null) { System.out.println("Response headers:");
		 * for (Header header : responseHeaders) { System.out.println("\t" +
		 * header.getName() + ": " + header.getValue()); } }
		 */
		/*
		 * System.out.println("=====CouchBase========");
		 *
		 *
		 *
		 *
		 * try { this.couchbaseClient.set(weburl, gson.toJson(doc)).get(); }
		 * catch (Exception ex) { ex.printStackTrace(); } // Return the result
		 * and cast it to string String result = (String)
		 * couchbaseClient.get(weburl); System.out.println(result);
		 *
		 * System.out.println("=============");
		 */
	}

	public boolean makeCareerInfoByTableTag(String url, String companyName)
			throws Exception {
		Source source = new Source(new URL(url));
		Element resultElement = null;
		CareerSiteInfo info = new CareerSiteInfo();

		List<Element> elementList = source
				.getAllElements(HTMLElementName.TABLE);

		for (Element element : elementList) {

			List<Element> trList = element.getAllElements(HTMLElementName.TR);
			for (Element tr : trList) {
				Element trElement = tr;
				List<Element> thList = trElement
						.getAllElements(HTMLElementName.TH);
				List<Element> tdList = trElement
						.getAllElements(HTMLElementName.TD);
				if (thList == null || thList.size() == 0) {
					continue;
				}
				if (tdList == null || tdList.size() == 0) {
					continue;
				}

				info.setUrl(url);
				String th = thList.get(0).getTextExtractor().toString();
				resultElement = tdList.get(0);
				String desc = resultElement.getTextExtractor()
						.setIncludeAttributes(false).toString();

				System.out.println("th:" + th);
				System.out.println("desc:" + desc);

				// 職種
				if (th.indexOf(CrawwleConstants.SHOKUSHU) >= 0) {
					info.setSyokusyu(desc);
				}
				// 勤務地
				else if (th.indexOf(CrawwleConstants.KINMUCHI) >= 0) {
					info.setKinnmuchi(desc);
				}
				// 年収、給与
				else if (th.indexOf(CrawwleConstants.NENNSYU) >= 0
						|| th.indexOf(CrawwleConstants.KYUUYO) >= 0) {
					info.setNensyu(desc);
				}
				// 仕事内容
				else if (th.indexOf(CrawwleConstants.WORK_DESC) >= 0) {
					info.setWorkDesc(desc);
				}
				// 雇用形態
				else if (th.indexOf(CrawwleConstants.KOYOU_KEITAI) >= 0) {
					info.setKoyouKeitai(desc);
				}
				// 勤務時間
				else if (th.indexOf(CrawwleConstants.WORK_TIME) >= 0) {
					info.setWorkTime(desc);
				} else {
					// return ;
				}

			}
		}
		if (info.getSyokusyu() == null && info.getNensyu() == null) {
			return false;
		}

		// 作成時間
		info.setMakeTime(Calendar.getInstance().getTime());
		info.setCompanyName(companyName);

		Gson gson = new Gson();
		getCouchbaseClient().set(companyName, gson.toJson(info)).get();
		return true;
	}

	public void makeCareerInfoByLiTag(String url, String domain)
			throws Exception {

		Source source = new Source(new URL(url));
		Gson gson = new Gson();

		CareerSiteInfo info;

		List<Element> elementList = source.getAllElements(HTMLElementName.LI);

		for (Element element : elementList) {

			String style = element.getAttributeValue("class");
			if (style == null || !getUrlMap().containsKey(style)) {
				continue;
			}

			List<Element> herfList = element.getAllElements(HTMLElementName.A);
			if (herfList == null || herfList.size() == 0) {
				continue;
			}

			info = new CareerSiteInfo();
			String aText = herfList.get(0).getTextExtractor().toString();
			if (aText == null || "".equals(aText)) {
				continue;
			}
			//String[] tList = aText.split(" ", 2);

			info.setCompanyName(aText);

			/*if(tList.length  > 1){
				info.setSyokusyu(tList[1]);
			}*/
			String link = herfList.get(0).getAttributeValue("href");
			if (link.indexOf(domain) >= 0) {
				info.setUrl(link);
			} else if(link.startsWith("/")) {
				info.setUrl("http://" + domain + link);
			}else if(url.substring(0,url.lastIndexOf("/")).length()>6){
				info.setUrl(url.substring(0,url.lastIndexOf("/") + 1)  + link);
			}else if(url.substring(0,url.lastIndexOf("/")).length()==6){
				info.setUrl(url.substring(0,url.lastIndexOf("/") + 1)  + link);
			}else{
				continue;
			}
			StringBuffer strBuf = new StringBuffer();
			List<Element> liList = element.getAllElements(HTMLElementName.LI);
			if(liList.size() > 0) {
				for (Element li : liList) {
					strBuf.append(li.getTextExtractor() + "　");
				}
			}else{
				strBuf.append( element.getTextExtractor()
					.setIncludeAttributes(false).toString());
			}
			String desc = strBuf.toString();
			if (desc == null || desc.length() == 0) {
				continue;
			}
			info.setWorkDesc(desc);

			// 作成時間
			info.setMakeTime(Calendar.getInstance().getTime());

			// return info;
			// result = (String) couchbaseClient.get;
			// System.out.println("=====CouchBase========");
			getCouchbaseClient().set(info.getUrl(), gson.toJson(info)).get();

			// Return the result and cast it to string
			String result = (String)
					getCouchbaseClient().get(info.getUrl());
			System.out.println(result);

			// Shutdown the client
			// System.out.println("=============");

		}
	}

	public void makeCareerInfoByTag(String tag, String url, String domain)
			throws Exception {

		Source source = new Source(new URL(url));
		Gson gson = new Gson();

		CareerSiteInfo info;

		List<Element> elementList = source.getAllElements(tag);

		for (Element element : elementList) {
			//TODO 解析ロジックをよくする、難しいところです。
			String style = element.getAttributeValue("class");
			if (style == null || !getUrlMap().containsKey(style)) {
				continue;
			}

			List<Element> herfList = element.getAllElements(HTMLElementName.A);
			if (herfList == null || herfList.size() == 0) {
				continue;
			}

			info = new CareerSiteInfo();
			info.setCompanyName(herfList.get(0).getTextExtractor().toString());
			String link = herfList.get(0).getAttributeValue("href");
			if (link.indexOf(domain) >= 0) {
				info.setUrl(link);
			} else if(link.startsWith("/")) {
				info.setUrl("http://" + domain + link);
			}else if(url.substring(0,url.lastIndexOf("/")).length()>6){
				info.setUrl(url.substring(0,url.lastIndexOf("/") + 1)  + link);
			}else if(url.substring(0,url.lastIndexOf("/")).length()==6){
				info.setUrl(url.substring(0,url.lastIndexOf("/") + 1)  + link);
			}else{
				continue;
			}

			if (info.getCompanyName() == null) {
				continue;
			}

			StringBuffer strBuf = new StringBuffer();
			List<Element> liList = element.getAllElements(HTMLElementName.LI);
			if(liList.size() > 0) {
				for (Element li : liList) {
					strBuf.append(li.getTextExtractor() + "　");
				}
			}else{
				strBuf.append( element.getTextExtractor()
					.setIncludeAttributes(false).toString());
			}
			String desc = strBuf.toString();
			if (desc == null || desc.length() == 0) {
				continue;
			}

			//TODO:整形
			desc = desc.replace(info.getCompanyName(), "");
			if (desc == null || desc.length() == 0) {
				continue;
			}
			info.setWorkDesc(desc);
			// 作成時間
			info.setMakeTime(Calendar.getInstance().getTime());

			getCouchbaseClient().set(info.getUrl(), gson.toJson(info)).get();
			// Return the result and cast it to string
			String result = (String) getCouchbaseClient().get(info.getUrl());
			System.out.println(result);

			// Shutdown the client
			// System.out.println("=============");

		}

	}

}
